package org.nerve.webmagic.processor;

import org.jsoup.nodes.Document;
import org.nerve.webmagic.DeepLimitScheduler;
import org.nerve.webmagic.bean.WebPage;
import org.nerve.webmagic.utils.DirUtil;
import org.nerve.webmagic.utils.ImageUtil;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import us.codecraft.webmagic.Page;
import us.codecraft.webmagic.Request;
import us.codecraft.webmagic.Site;
import us.codecraft.webmagic.processor.PageProcessor;
import us.codecraft.webmagic.utils.UrlUtils;

import java.io.IOException;
import java.nio.file.Paths;
import java.util.List;

/**
 * org.nerve.webmagic.spider.processor
 * Created by zengxm on 2016/4/20 0020.
 */
public class BaiduWenkuProcessor implements PageProcessor {
	private static Logger logger= LoggerFactory.getLogger(BaiduWenkuProcessor.class);

	private String homePage;
	private Site site;
	private String screenDirectory; //截图保存的目录
	private boolean keepScreenshotData =true;//是否保存截图元数据（base64）
	private boolean saveSccreen=false;  //是否将文件保存到本地文件

	private static final String domain="http://baike.baidu.com/";

	private ImageUtil imageUtil;

	public BaiduWenkuProcessor(){
		this(domain);
	}
	public BaiduWenkuProcessor(String page){
		this.homePage=page;

		this.imageUtil=new ImageUtil();
		if(screenDirectory==null){
			screenDirectory= DirUtil.PATH+"/screenShots/"+UrlUtils.getDomain(homePage);
		}
	}

	public BaiduWenkuProcessor setScreenDirectory(String screenDirectory) {
		this.screenDirectory = screenDirectory;
		return this;
	}
	public String getHomePage() {
		return homePage;
	}
	public String getScreenDirectory() {
		return screenDirectory;
	}
	public boolean isKeepScreenshotData() {
		return keepScreenshotData;
	}

	/**
	 * 是否保存截图元数据（base64）
	 * 当保存了截图到文件后，会用到这个字段判断是否清空元数据（清空page中的field）
	 * @param keepScreenshotData
	 * @return
	 */
	public BaiduWenkuProcessor setKeepScreenshotData(boolean keepScreenshotData) {
		this.keepScreenshotData = keepScreenshotData;
		return this;
	}
	public BaiduWenkuProcessor setSaveSccreen(boolean saveSccreen) {
		this.saveSccreen = saveSccreen;
		return this;
	}

	public void process(Page page) {
		Document doc=page.getHtml().getDocument();
		page.putField("title", doc.title());
		page.putField("url", page.getUrl());
		page.putField("html",  doc.toString());

		//判断是否有截图数据
		String screenData=page.getResultItems().get(WebPage.SCREENSHOT);
		if(screenData!=null){
			try {
				String screenshotPath=Paths.get(screenDirectory, System.currentTimeMillis()+".png").toString();
				logger.info("saving screenshot to "+screenshotPath);
				if(imageUtil.base64StringToImgAndSave(screenData, screenshotPath)){
					if(!keepScreenshotData)
						page.putField(WebPage.SCREENSHOT,null);
					page.putField("screenShotPath", screenshotPath);
				}
			} catch (IOException e) {
				logger.error("error on save screenshot:{}", e.getMessage());
			}
		}

		Object level=page.getRequest().getExtra(DeepLimitScheduler.LEVEL);
		int curLevel=level==null?0:((Integer)level);

		List<String> links=page.getHtml().links()
				//网易新闻格式：http://news.163.com/{年份}/{日期}/{小时}/
//				.regex(domain+"*")
				.regex(".*baike.baidu.com/.*")
				.all();

//		page.getHtml().links().all().forEach(l-> System.out.println(l));
		logger.info("title={},links size={}",doc.title(),links.size());
//		links.forEach(l-> System.out.println(l));

		//构建下一个深度的Request
		links.forEach(l-> page.addTargetRequest(
				new Request(l).putExtra(DeepLimitScheduler.LEVEL, curLevel+1)));
	}

	public Site getSite() {
		if(site==null){
			site=Site.me()
				.setRetryTimes(3)
				.setUserAgent("Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/47.0.2526.80 Safari/537.36")
				.setDomain(UrlUtils.getDomain(homePage))
				.addHeader("Connection","keep-alive")
				.addHeader("Accept","text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8")
//				.setCharset("gbk")
				.setSleepTime(2000);
		}

		return site;
	}
}
